"""
爬取虎牙颜值区所有房间数据
"""
from selenium import webdriver
import time
import csv
from pymongo import MongoClient

class DouYu(object):
    def __init__(self):
        # 创建驱动对象
        self.driver = webdriver.Chrome()
        # 指定访问的网址
        self.url = self.driver.get("https://www.douyu.com/g_dance")
        # 窗口最大化
        self.driver.maximize_window()
        # 存放csv数据的大列表
        # self.csv_list = [["昵称","描述","观看人数","封面链接","房间链接"],]

        # 初始化mongo对象
        self.cli = MongoClient(host="127.0.0.1",port=27017)
        # 连接某张表
        self.douyu = self.cli["wxt"]["douyu"]
        # 写入mongodb中的大列表
        self.mongo_list = []

        """
        [
            {
            name:xxx,
            des:xxx,
            count:xxx,
            url:xxx,
            room_url:xxx
                },
            {
                },
            {
                },
        
        ]
        """
    # 提取数据的方法
    def get_data(self):
        time.sleep(3)
        flag = 1000
        # 执行js 滚动到底部
        for i in range(8):
            js = f"var q=document.documentElement.scrollTop={flag};"
            self.driver.execute_script(js)
            time.sleep(2)
            flag += 1000
        # 获取所有的房间标签对象
        # rooms = self.driver.find_elements_by_xpath("//*[@id='listAll']/section[2]/div[2]/ul/li")
        rooms = self.driver.find_elements_by_xpath("//*[@id='listAll']/div[2]/ul/li")
        print(rooms)
        # 遍历所有的房间
        for room in rooms:
            # csv需要的列表
            # room_list = []

            # mongo需要的字典
            room_dict = {}

            # 获取主播的昵称
            name = room.find_element_by_xpath(".//div[@class='DyListCover-userName']").text
            # 房间描述
            desc = room.find_element_by_xpath(".//h3[@class='DyListCover-intro']").text
            # 观看人数
            see_count = room.find_element_by_xpath(".//span[@class='DyListCover-hot']").text
            # 封面图片链接
            try:
                img = room.find_element_by_xpath(".//div[@class='DyListCover-imgWrap']/div//img").get_attribute("src")
            except:
                img = ""
            print(img)
            # 房间链接
            path = room.find_element_by_xpath("./div/a").get_attribute("href")
            # 写入csv
            # room_list.extend((name,desc,see_count,img,path))
            # self.csv_list.append(room_list)
            room_dict["name"] = name
            room_dict["desc"] = desc
            room_dict["see_count"] = see_count
            room_dict["img"] = img
            room_dict["path"] = path
            print(room_dict)
            self.mongo_list.append(room_dict)

        # 将csv的大列表返回
        # return self.csv_list

        # 将mongo的大列表返回
        return self.mongo_list


    # 循环数据
    def click_next(self):
        while True:
            # 获取数据  data为get_data()的返回值 就是大列表
            data = self.get_data()
            # 找到下一页的按钮
            next_page = self.driver.find_element_by_xpath("//*[@id='listAll']/div[2]/div/ul/li[4]")
            # 判断li标签的属性aria-disabled 的值到底是true 还是false
            # 如果是 true 代表没有下一页数据 就不用点了 直接退出
            # 如果是false  代表还有下一页数据 可以点击
            if next_page.get_attribute("aria-disabled") == "false":
                next_page.click()
            else:
                break
        return data
    # 保存csv
    # def save_csv(self,data):
    #     with open("zhubo.csv",'a',encoding="utf-8",newline="") as f:
    #         for x in data:
    #             writter = csv.writer(f)
    #             writter.writerow(x)

    # 保存mongodb数据库
    def save_mongo(self,data):
        print(data)
        self.douyu.insert_many(data)



    # run方法
    def run(self):
        # 开始爬取数据
        data = self.click_next()
        # 写入csv
        # self.save_csv(data)
        # 写入到mongo
        self.save_mongo(data)
        # 退出浏览器
        self.driver.quit()


if __name__ == '__main__':
    huya = DouYu()
    huya.run()




